# -*- coding: utf-8 -*-
import scrapy
from scrapys.items import TestItem
from collections import OrderedDict
import math
import datetime


class TestSpider(scrapy.Spider):
    name = 'test'
    allowed_domains = ['chinadrugtrials.org.cn']
    # custom_settings = {
    #     'LOG_LEVEL':'INFO',
    #     'ITEM_PIPELINES' : {'trialscr.pipelines.TrialscrPipeline': 300}
    # }

    def __init__(self, ckm_index=1, end=3, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        self.ckm_index = int(ckm_index)
        self.end = int(end)

    def start_requests(self):
        self.start_urls = 'http://www.chinadrugtrials.org.cn/eap/clinicaltrials.searchlistdetail'

        for i in range(self.ckm_index, self.end):
            currentpage = math.ceil(i/20)
            self.data = {
                'ckm_index': str(i),
                'currentpage': str(currentpage),
                'pagesize': '20',
                'rule': 'CTR',
                'sort': 'desc',
                'sort2': 'desc'
            }
            self.ckm_index = i
            self.currentpage = currentpage
            yield scrapy.FormRequest(url = self.start_urls, method = 'POST',
                formdata = self.data, callback = self.parse, dont_filter = True
            )
    def getContent(self, item):
        return [i.strip() for i in item]

    def parse(self, response):
        self.item = TestItem()
        self.item['origin'] = 'CDE' # 来源	CDE
        self.item['url'] = self.start_urls	# 网址
        lis1 = response.xpath('//div[@class="apply_zhgl"]/table')
        # for i in lis1[0]:
        item = lis1[0].xpath("./tr/td//text()").extract()
        print([i.strip() for i in item])
            # print(self.getContent(item))

            # print(lis1[0].xpath("./tr//text()").extract())
        print("````````````````````````````````````````````````````````\n")

        # print(lis1.xpath("./tr//text()").extract())
        # self.item['status'] = lis1[0].strip() # 状态

        # lis = response.xpath('//div[@id="div_open_close_01"]/table[1]/tr/td//text()').extract()
        # content = [i.strip() for i in lis]
        # self.item['regno'] = content[1]    # 登记号
        # self.item['indication'] = content[3] # 适应症
        # self.item['pub_title'] = content[5] # 试验通俗题目
        # self.item['sci_title'] = content[7] # 试验专业题目
        # self.item['product'] = content[13] # 药物名称
        # try:
        #     ckm_index = response.xpath("/html/body/div/table/tbody/tr[2]/td/div[2]/div/div[2]/div[1]/div[2]//text()").extract()
        #     self.getTopics(response) # 题目和背景信息
        #     self.getClinical(response) # 临床实验信息
        #     self.getSponsor(response) # 申办者信息
        #     self.getInvesInfo(response) # 主要研究信息
        #     self.getgroups(response) # 实验分组
        #     self.tag_ckm_index("成功获取--ckm_index:" + str(ckm_index[8]) + " --regno:" + self.item['regno'])
        #     pass
        # except Exception as e:
        #     self.tag_ckm_index("异常---" + str(self.ckm_index))
        #     raise
        # else:
        #     print(self.item['regno'])
        #     yield self.item

    def getgroups (self, response):
        # 实验分组
        keys = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr[position()<16 and position()>13]/td[1]//text()').extract()
        keys = [i.strip() for i in keys]

        test_tag = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr[14]/td[2]/table/tr[position()>1]//text()').extract()
        compare_tag = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr[15]/td[2]/table/tr[position()>1]//text()').extract()

        testItem = self.getgroup(test_tag)
        compareItem = self.getgroup(compare_tag)

        self.item['groups'] = {keys[0]:testItem, keys[1]:compareItem}

        pass

    def getgroup(self, items):
        items = [i.strip() for i in items if i.strip() != '']
        item = []
        for i in range (len(items)):
            single = OrderedDict()  # 有序字典
            if i%3 == 0:
                single['code'] = items[i]
                single['group'] = items[i+1]
                single['use'] = items[i+2]
                item.append(single)
        return item
        pass

    def getTopics(self, response):
        lis1 = response.xpath('//div[@id="div_open_close_01"]/table[8]/tr/td//text()').extract()
        self.item['status'] = lis1[0].strip() # 状态

        lis = response.xpath('//div[@id="div_open_close_01"]/table[1]/tr/td//text()').extract()
        content = [i.strip() for i in lis]
        self.item['regno'] = content[1]    # 登记号
        self.item['indication'] = content[3] # 适应症
        self.item['pub_title'] = content[5] # 试验通俗题目
        self.item['sci_title'] = content[7] # 试验专业题目
        self.item['product'] = content[13] # 药物名称

        pass

    def getSponsor(self, response):
        lis = response.xpath('//div[@class="cxtj_tm"]/table/tr/td//text()').extract()
        lis = [i.strip() for i in lis]
        self.item['sponsor'] = lis[9] # 申办者名称
        self.item['sponsor_contact'] = lis[5] #申办者联系人
        self.item['time'] = lis[7] #首次公示信息日期

        lis = response.xpath('//div[@id="div_open_close_01"]/table[2]/tr[5]/td[2]/text()').extract()
        self.item['fee_source'] = lis[0].strip()	# 试验项目经费来源

        pass

    def getClinical (self, response):
        lis = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr/td//text()').extract()
        content = [i.strip() for i in lis if i.strip() !='']
        self.item['purpose'] = content[1] # 研究目的
        self.item['type'] = content[4] # 实验分类
        self.item['phase'] = content[6] # 实验分期
        self.item['design'] = content[8] # 设计类型
        self.item['random'] = content[10] # 随机化
        self.item['blind'] = content[12] # 盲法
        self.item['volunteer'] = content[21] # 健康受试者
        self.item['age'] = content[17] # 年龄
        ages = re.findall('\d+', content[17])

        if len(ages) == 2:
            self.item['age_lower'] = ages[0]	# 最小
            self.item['age_upper'] = ages[1]	# 最大
        elif len(ages) == 1:
            self.item['age_lower'] = ages[0]	# 最小
            self.item['age_upper'] = ''	# 最大
        else:
            self.item['age_lower'] = ''	# 最小
            self.item['age_upper'] = ''	# 最大
        self.item['sex'] = content[19] # 性别

        # 受试者信息-入选标准
        incs = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr[9]/td[2]//text()').extract()
        self.getIncAndExc(incs, 'inc')
        # 受试者信息-排除标准
        excs = response.xpath('//div[@id="div_open_close_01"]/table[3]/tr[10]/td[2]//text()').extract()
        self.getIncAndExc(excs, 'exc') # 排除标准
        pass

    def getInvesInfo (self, response):
        self.getInstitution(response) # 获取机构信息
        self.getMajorInves(response) # 获取主要研究者信息

        pass

    def getInstitution (self, response):
        lis = response.xpath('//*[@id="hspTable"]//text()').extract()
        lis = [i.strip() for i in lis]
        item = []
        for i in range (len(lis[15:])):
            single = OrderedDict()  # 有序字典
            if i%14 == 0:
                single['code'] = lis[15:][i+1]
                single['hospital'] = lis[15:][i+3] # 机构名称
                single['major_leader'] = lis[15:][i+5] # 主要研究者
                single['country'] = lis[15:][i+7] # 国家
                single['province'] = lis[15:][i+9] # 省
                single['city'] = lis[15:][i+11] # 城市
                item.append(single)
        self.item['sites'] = item

    def getMajorInves (self, response):
        lis = response.xpath('//*[@id="div_open_close_01"]/table[6]/tr[2]/td//text()').extract()
        lis = [ i.strip() for i in lis]
        item = []
        for i in range (len(lis[1:])):
            single = OrderedDict()  # 有序字典
            if i%36 == 0:
                 single['tudy_leader'] = lis[i+4].strip() # 姓名
                 single['position'] = lis[i+8].strip()  # 职称
                 single['phone'] = lis[i+13].strip()  # 电话
                 single['Email'] = lis[i+17].strip()  # Email
                 single['leader_hosp_address'] = lis[i+23].strip()  # 地址
                 item.append(single)

        self.item['major_leader'] = item

    def getIncAndExc (self, items, type):
        items = [ i.strip() for i in items if i.strip() != '']

        single = OrderedDict()  # 有序字典
        if len(items):
            for i in range (len(items)):
                if i%2 == 0:
                    single[items[i]] = items[i+1]

            self.item[type] = single
            pass
        else:
            pass

    def tag_ckm_index (self, ckm_index):
        with open("/var/www/html/trialscr/log/ckm_index.txt", 'a+', encoding='utf-8') as f:
            time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            f.write(time + ":" + ckm_index +"\n")
            f.close()
